#!/bin/sh

echo "Fetch up-to-date NASDAQ100, SP500 and Barron400 Stocks List. Join them to make CompositeStocks as stock Universe"
curl -stderr "http://www.datawm.com/nastocks/nyse.txt" |cut -d',' -f1 > NYSE
#SP 500
wget -q "http://www2.standardandpoors.com/app/Satellite?pagename=spcom/page/download&sectorid= > '00'&itemname=< 'Z'&dt=25-OCT-2007&indexcode=500" -O tmp$$
cat tmp$$ |awk '{print $1}' |sort |uniq > SP500Stocks
#NAZ 100
curl -stderr "http://dynamic.nasdaq.com/dynamic/nasdaq100_activity.stm" |egrep -o ">[A-Z]{1,5}<" |sed -e 's/>//g' -e 's/<//g' > NAZ100Stocks
#Barron 400
curl -stderr "http://www.marketgrader.com/MG.Services/Barrons400/components.jsp" |grep "<td>" |grep -v tr |grep -v img  |cut -d'>' -f2 |cut -d'<' -f1 > Barron400Stocks
cat SP500Stocks NAZ100Stocks Barron400Stocks |sort |uniq > CompositeStocks
echo "Fetch Top 100 liquid ETF"
curl -stderr "http://finance.yahoo.com/etf/browser/tv?c=0&k=5&f=0&o=d&cs=1&ce=100" |sed 's/tr/\n/g' |egrep -o "q?s=[A-Z]+" |sort |uniq |cut -d'=' -f2 > tmp$$
>ETFStocks
cat tmp$$ |while read line
do
    curl -stderr "http://download.finance.yahoo.com/d/quotes.csv?s=$line&f=sn" |sed 's/"//g' >> ETFStocks
done

